package com.fileshare.web.utils;

import com.huaban.analysis.jieba.JiebaSegmenter;
import lombok.SneakyThrows;
import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;

import java.io.*;
import java.nio.charset.StandardCharsets;
import java.util.*;

public class TextClassifyUtil {
    //邮件索引
    private static final String MAIL_INDEX_PATH = "F:\\数据集\\trec06c (1)\\trec06c (1)\\trec06c\\full/index";
    //垃圾邮件分词表
    private static final Map<String, Integer> GOOD_WORDS = new HashMap<>();
    //正常邮件分词表
    private static final Map<String, Integer> BAD_WORDS = new HashMap<>();
    //垃圾邮件分词概率表
    private static final Map<String, Double> RATE_MAP = new HashMap<>();
    //
    private static final JiebaSegmenter JIE_BA_SPLITTER = new JiebaSegmenter();
    //
    private static final IKSegmenter IK_SPLITTER = new IKSegmenter(new StringReader(""), true);
    //
    private static final Set<String> STOP_WORDS = new HashSet<>();
    private static final Set<String> VOCAB = new HashSet<>();

    public static void main(String[] args) throws IOException {
//        Path path = Paths.get(new File("C:\\Users\\geticsen\\Desktop\\test.txt").getPath());
//        WordDictionary.getInstance().loadUserDict(path);
//        ik();
//        System.out.println();
//        jieba();
        testBase();
    }

    public static void ik() {
        String text = "基于java语言开发的轻量级的中文分词工具包";
        List<String> words = splitLineWithIk(text);
        System.out.print("IK: ");
        for (String str : words) {
            System.out.print(str + " ");
        }
    }

    public static void jieba() {
        String text = "基于java语言开发的轻量级的中文分词工具包";
        System.out.print("jieba: ");
        List<String> words = splitLineWithJieBa(text);
        for (String str : words) {
            System.out.print(str + " ");
        }
    }

    private static void addStopWords() {
        try (BufferedReader br = new BufferedReader(new InputStreamReader(
                new FileInputStream(new File("F:\\数据集\\SpeechDetection-master\\SpeechDetection-master\\Chinese-StopWords-master\\Chinese-StopWords-master\\ChineseStopWords.txt")), StandardCharsets.UTF_8))) {
            while (true) {
                String s = br.readLine();
                if (s == null) break;
                if (!s.equals("") && !s.trim().equals("")) {
                    STOP_WORDS.add(s);
                }
            }
        } catch (Exception ignored) {
        }
    }

    private static void save() {
        saveList(STOP_WORDS, "stop_words.txt");
        saveList(VOCAB, "vocab.txt");
        saveMap(GOOD_WORDS, "good_words.txt");
        saveMap(BAD_WORDS, "bad_words.txt");
    }

    private static void saveList(Collection<String> list, String file) {
        try {
            BufferedWriter out = new BufferedWriter(new FileWriter("F:\\数据集\\TextClassify\\" + file));
            for (String word : list) {
                out.write(word + "\n");
            }
            out.close();
            System.out.println("写入完成");
        } catch (IOException ignored) {
        }
    }

    private static void saveMap(Map<String, ? extends Number> map, String file) {
        try {
            BufferedWriter out = new BufferedWriter(new FileWriter("F:\\数据集\\TextClassify\\" + file));
            for (Map.Entry<String, ? extends Number> entry : map.entrySet()) {
                String line = entry.getKey() + ":" + entry.getValue() + "\n";
                out.write(line);
            }
            out.close();
            System.out.println("写入完成");
        } catch (IOException ignored) {
        }
    }

    private static void addVocab() {
        List<String> paths = new ArrayList<>();
        String base = "F:\\数据集\\SpeechDetection-master\\SpeechDetection-master\\source\\敏感词汇\\";
        paths.add(base + "暴恐词库.txt");
        paths.add(base + "反动词库.txt");
        paths.add(base + "民生词库.txt");
        paths.add(base + "其他词库.txt");
        paths.add(base + "色情词库.txt");
        paths.add(base + "贪腐词库.txt");
        for (String filePath : paths) {
            try (BufferedReader br = new BufferedReader(new InputStreamReader(
                    new FileInputStream(new File(filePath)), "gbk"))) {
                while (true) {
                    String s = br.readLine();
                    if (s == null) break;
                    if (!s.equals("") && !s.trim().equals("")) {
                        VOCAB.add(s);
                        BAD_WORDS.put(s, BAD_WORDS.containsKey(s) ? BAD_WORDS.get(s) + 1 : 100);
                    }
                }
            } catch (Exception ignored) {
            }
        }
        org.wltea.analyzer.dic.Dictionary.getSingleton().addWords(VOCAB);
    }

    private static void testBase() throws IOException {
        addStopWords();
        addVocab();
        initWordMap1();
        initWordMap();
        dealWithRateMap();
        Scanner scanner = new Scanner(System.in);
        String line = "";
        while (!"quit".equals(line)) {
            System.out.print("请输入一行: ");
            line = scanner.nextLine();
            System.out.println(test(line));
        }
        save();
    }

    private static String test(String line) {
        List<String> words = splitLineWithJieBa(line);
        double normalRate = 1d;
        double unNormalRate = 1d;
        double possible = 0d;
        for (String word : words) {
            if (RATE_MAP.containsKey(word)) {
                double tmp = RATE_MAP.get(word);
                unNormalRate *= 1d - tmp;
                normalRate *= tmp;
                if (unNormalRate == 0d && normalRate == 0d) {
                    //同时趋于0,则概率设为0.5 已经同时趋于0，不需要继续遍历
                    possible = 0.5d;
                    break;
                } else if (unNormalRate == 0d) {//正常邮件概率趋于0
                    possible = 1d;
                    if (normalRate > Math.pow(10, -200)) {//差125量级，直接判定
                        break;
                    }
                } else if (normalRate == 0d) {//垃圾邮件趋于0
                    possible = 0d;
                    if (unNormalRate > Math.pow(10, -200)) {//差125量级，直接判定
                        break;
                    }
                }
            }
        }
        System.out.println("possible = " + possible);
        System.out.println("rate = " + normalRate);
        System.out.println("tempRate = " + unNormalRate);
        return (normalRate > unNormalRate && possible < 0.5) ? "正常" : "垃圾";
    }

    private static List<String> getWordList(String index) {
        index = index.replace("..", "F:\\数据集\\trec06c (1)\\trec06c (1)\\trec06c");
        String str = readFile(index);
        return splitLineWithIk(str);
    }

    private static List<String> splitLineWithJieBa(String line) {
        List<String> tempList = JIE_BA_SPLITTER.sentenceProcess(line);
        List<String> rightList = new ArrayList<>();
        for (String s : tempList) {
            //过滤掉标点符号等无意义分词
            if (!STOP_WORDS.contains(s)) {
                rightList.add(s);
            }
        }
        return rightList;
    }

    @SneakyThrows
    private static List<String> splitLineWithIk(String line) {
        StringReader input = new StringReader(line);
        IK_SPLITTER.reset(input);
        Lexeme lex = null;
        List<String> words = new ArrayList<>();
        while ((lex = IK_SPLITTER.next()) != null) {
            String lexemeText = lex.getLexemeText();
            if (!STOP_WORDS.contains(lexemeText)) {
                words.add(lexemeText);
            }
        }
        input.close();
        return words;
    }

    public static void initWordMap1() throws IOException {
        String fileName = "F:\\数据集\\online_shopping_10_cats.csv";
        try (BufferedReader bufferedReader = new BufferedReader(new FileReader(fileName))) {
            //得到训练集概率表
            String line = bufferedReader.readLine();
            System.out.println("-----------------------------------获取数据中----------------------------------");
            while ((line = bufferedReader.readLine()) != null) {
                String[] split = line.split(",");
                List<String> words = splitLineWithIk(split[2]);
                processWords(split[1], words);
            }
        } catch (Exception ignored) {

        }
    }

    public static void initWordMap() throws IOException {
        long trainStartTime = System.currentTimeMillis();
        try (BufferedReader bufferedReader = new BufferedReader(new FileReader(MAIL_INDEX_PATH))) {
            //得到训练集概率表
            String indexLine;
            System.out.println("-----------------------------------获取数据中----------------------------------");
            while ((indexLine = bufferedReader.readLine()) != null) {
                //分离出邮件真实类型和文件名索引
                String[] typeAndPath = indexLine.split(" ");
                String type = typeAndPath[0];
                List<String> wordList = getWordList(typeAndPath[1]);
                processWords(type, wordList);
            }
            long trainFinishTime = System.currentTimeMillis();
            System.out.println("数据分词时间 = " + (trainFinishTime - trainStartTime) / 1000);
        } catch (Exception ignored) {

        }

    }

    private static void processWords(String type, List<String> wordList) {
        if (type.equals("spam") || type.equals("1")) {
            for (String s : wordList) {
                GOOD_WORDS.put(s, GOOD_WORDS.containsKey(s) ? GOOD_WORDS.get(s) + 1 : 1);
            }
        } else {
            for (String s : wordList) {
                BAD_WORDS.put(s, BAD_WORDS.containsKey(s) ? BAD_WORDS.get(s) + 1 : 1);
            }
        }
    }

    public static void dealWithRateMap() {
        int goodSize = GOOD_WORDS.size();
        int badSize = BAD_WORDS.size();
        for (Map.Entry<String, Integer> entry : GOOD_WORDS.entrySet()) {
            String key = entry.getKey();
            double rate = (double) entry.getValue() / goodSize;
            double allRate = rate;
            Integer counter = BAD_WORDS.get(key);
            if (counter != null) {
                allRate += (double) counter / badSize;
            } else {
                //如果正常邮件中未出现该词，则默认在正常邮件中出现过1次，防止判断过于绝对
                allRate += 1d / (double) BAD_WORDS.size();
            }
            RATE_MAP.put(key, rate / allRate);
        }
    }

    /**
     * 读文件
     */
    public static String readFile(String filePath) {
        StringBuilder str = new StringBuilder();
        try (BufferedReader br = new BufferedReader(new InputStreamReader(
                new FileInputStream(new File(filePath)), "gbk"))) {
            String tmp = "";
            int flag = 0;
            while ((tmp = br.readLine()) != null) {
                if (flag == 1) {
                    str.append(tmp);
                }
                if (tmp.equals("") || tmp.length() == 0) {
                    flag = 1;
                }
            }
        } catch (Exception ignored) {
        }
        return str.toString();
    }


}
